{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import math\n",
    "import time\n",
    "import jieba\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from gensim.models import word2vec\n",
    "from gensim import models\n",
    "from langconv import *\n",
    "\n",
    "%matplotlib inline\n",
    "pd.options.display.max_columns = None\n",
    "plt.rcParams['font.sans-serif']=['SimHei']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['其次', '，', '这样', '的', '花卉', '一', '如', '远', '嫁', '它', '乡', '的', '女子', '，', '离别', '了', '故园', '来到', '异地', '，', '若', '不', '能', '以', '日夜', '牵念', '之', '情', '而', '把守', '，', '寒暑', '不', '忘', '之', '心', '而', '敬爱', '，', '却', '希翼', '它', '日日', '保持', '婀娜', '纤巧', '之', '美貌', '，', '婆娑', '盘旋', '之', '旖丽', '，', '如何', '才', '是', '可以', '呢', '？']\n"
     ]
    }
   ],
   "source": [
    "cws_model = r'D:\\GitHub\\Pyltp\\ltp_data\\cws.model'\n",
    "\n",
    "from pyltp import Segmentor\n",
    "segmentor = Segmentor()\n",
    "segmentor.load(cws_model)\n",
    "words = segmentor.segment('其次，这样的花卉一如远嫁它乡的女子，离别了故园来到异地，若不能以日夜牵念之情而把守，寒暑不忘之心而敬爱，却希翼它日日保持婀娜纤巧之美貌，婆娑盘旋之旖丽，如何才是可以呢？')\n",
    "print([i for i in words])\n",
    "segmentor.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['其次', '，', '这样', '的', '花卉', '一如', '远嫁', '它', '乡', '的', '女子', '，', '离别', '了', '故园', '来到', '异地', '，', '若', '不能', '以', '日夜', '牵念', '之情', '而', '把守', '，', '寒暑', '不忘之心', '而', '敬爱', '，', '却', '希翼', '它', '日日', '保持', '婀娜', '纤巧', '之', '美貌', '，', '婆娑', '盘旋', '之旖丽', '，', '如何', '才', '是', '可以', '呢', '？']\n"
     ]
    }
   ],
   "source": [
    "print([i for i in jieba.cut('其次，这样的花卉一如远嫁它乡的女子，离别了故园来到异地，若不能以日夜牵念之情而把守，寒暑不忘之心而敬爱，却希翼它日日保持婀娜纤巧之美貌，婆娑盘旋之旖丽，如何才是可以呢？')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['其次', '，', '这样', '的', '花卉', '一如', '远嫁', '它', '乡', '的', '女子', '，', '离别', '了', '故园', '来到', '异地', '，', '若', '不', '能', '以', '日夜', '牵念', '之', '情', '而', '把守', '，', '寒暑', '不', '忘', '之', '心', '而', '敬爱', '，', '却', '希翼', '它', '日日', '保持', '婀娜', '纤巧', '之', '美貌', '，', '婆娑', '盘旋', '之', '旖丽', '，', '如何', '才', '是', '可以', '呢', '？']\n"
     ]
    }
   ],
   "source": [
    "import pkuseg\n",
    "\n",
    "seg = pkuseg.pkuseg()\n",
    "text = seg.cut('其次，这样的花卉一如远嫁它乡的女子，离别了故园来到异地，若不能以日夜牵念之情而把守，寒暑不忘之心而敬爱，却希翼它日日保持婀娜纤巧之美貌，婆娑盘旋之旖丽，如何才是可以呢？')\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 105000 entries, 0 to 104999\n",
      "Data columns (total 22 columns):\n",
      "id                                          105000 non-null int64\n",
      "content                                     105000 non-null object\n",
      "location_traffic_convenience                105000 non-null int64\n",
      "location_distance_from_business_district    105000 non-null int64\n",
      "location_easy_to_find                       105000 non-null int64\n",
      "service_wait_time                           105000 non-null int64\n",
      "service_waiters_attitude                    105000 non-null int64\n",
      "service_parking_convenience                 105000 non-null int64\n",
      "service_serving_speed                       105000 non-null int64\n",
      "price_level                                 105000 non-null int64\n",
      "price_cost_effective                        105000 non-null int64\n",
      "price_discount                              105000 non-null int64\n",
      "environment_decoration                      105000 non-null int64\n",
      "environment_noise                           105000 non-null int64\n",
      "environment_space                           105000 non-null int64\n",
      "environment_cleaness                        105000 non-null int64\n",
      "dish_portion                                105000 non-null int64\n",
      "dish_taste                                  105000 non-null int64\n",
      "dish_look                                   105000 non-null int64\n",
      "dish_recommendation                         105000 non-null int64\n",
      "others_overall_experience                   105000 non-null int64\n",
      "others_willing_to_consume_again             105000 non-null int64\n",
      "dtypes: int64(21), object(1)\n",
      "memory usage: 17.6+ MB\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 15000 entries, 0 to 14999\n",
      "Data columns (total 22 columns):\n",
      "id                                          15000 non-null int64\n",
      "content                                     15000 non-null object\n",
      "location_traffic_convenience                15000 non-null int64\n",
      "location_distance_from_business_district    15000 non-null int64\n",
      "location_easy_to_find                       15000 non-null int64\n",
      "service_wait_time                           15000 non-null int64\n",
      "service_waiters_attitude                    15000 non-null int64\n",
      "service_parking_convenience                 15000 non-null int64\n",
      "service_serving_speed                       15000 non-null int64\n",
      "price_level                                 15000 non-null int64\n",
      "price_cost_effective                        15000 non-null int64\n",
      "price_discount                              15000 non-null int64\n",
      "environment_decoration                      15000 non-null int64\n",
      "environment_noise                           15000 non-null int64\n",
      "environment_space                           15000 non-null int64\n",
      "environment_cleaness                        15000 non-null int64\n",
      "dish_portion                                15000 non-null int64\n",
      "dish_taste                                  15000 non-null int64\n",
      "dish_look                                   15000 non-null int64\n",
      "dish_recommendation                         15000 non-null int64\n",
      "others_overall_experience                   15000 non-null int64\n",
      "others_willing_to_consume_again             15000 non-null int64\n",
      "dtypes: int64(21), object(1)\n",
      "memory usage: 2.5+ MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"sentiment_analysis_trainingset.csv\")\n",
    "validation = pd.read_csv(\"sentiment_analysis_validationset.csv\")\n",
    "print(train.info())\n",
    "print(validation.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>location_traffic_convenience</th>\n",
       "      <th>location_distance_from_business_district</th>\n",
       "      <th>location_easy_to_find</th>\n",
       "      <th>service_wait_time</th>\n",
       "      <th>service_waiters_attitude</th>\n",
       "      <th>service_parking_convenience</th>\n",
       "      <th>service_serving_speed</th>\n",
       "      <th>price_level</th>\n",
       "      <th>price_cost_effective</th>\n",
       "      <th>price_discount</th>\n",
       "      <th>environment_decoration</th>\n",
       "      <th>environment_noise</th>\n",
       "      <th>environment_space</th>\n",
       "      <th>environment_cleaness</th>\n",
       "      <th>dish_portion</th>\n",
       "      <th>dish_taste</th>\n",
       "      <th>dish_look</th>\n",
       "      <th>dish_recommendation</th>\n",
       "      <th>others_overall_experience</th>\n",
       "      <th>others_willing_to_consume_again</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>\"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                            content  \\\n",
       "0   0  \"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，...   \n",
       "\n",
       "   location_traffic_convenience  location_distance_from_business_district  \\\n",
       "0                            -2                                        -2   \n",
       "\n",
       "   location_easy_to_find  service_wait_time  service_waiters_attitude  \\\n",
       "0                     -2                 -2                         1   \n",
       "\n",
       "   service_parking_convenience  service_serving_speed  price_level  \\\n",
       "0                           -2                     -2           -2   \n",
       "\n",
       "   price_cost_effective  price_discount  environment_decoration  \\\n",
       "0                    -2               1                      -2   \n",
       "\n",
       "   environment_noise  environment_space  environment_cleaness  dish_portion  \\\n",
       "0                 -2                 -2                    -2            -2   \n",
       "\n",
       "   dish_taste  dish_look  dish_recommendation  others_overall_experience  \\\n",
       "0          -2          1                   -2                          1   \n",
       "\n",
       "   others_willing_to_consume_again  \n",
       "0                               -2  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，大众点评给了我这个土老冒一个见识的机会。看介绍棒棒糖是用德国糖做的，不会很甜，中间的照片是糯米的，能食用，真是太高端大气上档次了，还可以买蝴蝶结扎口，送人可以买礼盒。我是先打的卖家电话，加了微信，给卖家传的照片。等了几天，卖家就告诉我可以取货了，去大官屯那取的。虽然连卖家的面都没见到，但是还是谢谢卖家送我这么可爱的东西，太喜欢了，这哪舍得吃啊。\"'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['content'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>location_traffic_convenience</th>\n",
       "      <th>location_distance_from_business_district</th>\n",
       "      <th>location_easy_to_find</th>\n",
       "      <th>service_wait_time</th>\n",
       "      <th>service_waiters_attitude</th>\n",
       "      <th>service_parking_convenience</th>\n",
       "      <th>service_serving_speed</th>\n",
       "      <th>price_level</th>\n",
       "      <th>price_cost_effective</th>\n",
       "      <th>price_discount</th>\n",
       "      <th>environment_decoration</th>\n",
       "      <th>environment_noise</th>\n",
       "      <th>environment_space</th>\n",
       "      <th>environment_cleaness</th>\n",
       "      <th>dish_portion</th>\n",
       "      <th>dish_taste</th>\n",
       "      <th>dish_look</th>\n",
       "      <th>dish_recommendation</th>\n",
       "      <th>others_overall_experience</th>\n",
       "      <th>others_willing_to_consume_again</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>\"哎，想当年来佘山的时候，啥都没有，三品香算镇上最大看起来最像样的饭店了。菜品多，有点太多，...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                            content  \\\n",
       "0   0  \"哎，想当年来佘山的时候，啥都没有，三品香算镇上最大看起来最像样的饭店了。菜品多，有点太多，...   \n",
       "\n",
       "   location_traffic_convenience  location_distance_from_business_district  \\\n",
       "0                            -2                                        -2   \n",
       "\n",
       "   location_easy_to_find  service_wait_time  service_waiters_attitude  \\\n",
       "0                     -2                  0                        -2   \n",
       "\n",
       "   service_parking_convenience  service_serving_speed  price_level  \\\n",
       "0                           -2                     -2            1   \n",
       "\n",
       "   price_cost_effective  price_discount  environment_decoration  \\\n",
       "0                    -2              -2                      -2   \n",
       "\n",
       "   environment_noise  environment_space  environment_cleaness  dish_portion  \\\n",
       "0                 -2                 -2                    -2            -2   \n",
       "\n",
       "   dish_taste  dish_look  dish_recommendation  others_overall_experience  \\\n",
       "0           0         -2                   -2                          1   \n",
       "\n",
       "   others_willing_to_consume_again  \n",
       "0                                0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"哎，想当年来佘山的时候，啥都没有，三品香算镇上最大看起来最像样的饭店了。菜品多，有点太多，感觉啥都有，杂都不足以形容。随便点些，居然口味什么的都好还可以，价钱自然是便宜当震惊。元宝虾和椒盐九肚鱼都不错吃。不过近来几次么，味道明显没以前好了。冷餐里面一个凉拌海带丝还可以，酸酸甜甜的。镇上也有了些别的大点的饭店，所以不是每次必来了。对了，这家的生意一如既往的超级好，不定位基本吃不到。不过佘山这边的人吃晚饭很早的，所以稍微晚点去就很空了。\"'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation['content'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2    11757\n",
       " 1     2925\n",
       "-1      182\n",
       " 0      136\n",
       "Name: location_traffic_convenience, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation.location_traffic_convenience.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1f05dbe80b8>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD3CAYAAADxJYRbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAQeUlEQVR4nO3df4wc91nH8feDnUjGl1ouibY4orUsWUilrkly0Jg61V5Iooa2CEIhEW5RFdBB1QZBi6irRhSqAikigdZSECe5aVSiWi6iTaO4aahgqQlOS66/3F+IH7qguiShcrB7Uf5p9PDHTmT3cr6bnd3bm9vv+yWdPPfM7He+z+34c3Mzu3eRmUiSyvBD6z0BSdL4GPqSVBBDX5IKYuhLUkEMfUkqiKEvSQXZvN4TWMmll16aO3fubPTYZ555hq1bt452QuvEXtpnUvoAe2mrYXqZn5//bmZetty6Vof+zp07eeyxxxo9ttfr0e12RzuhdWIv7TMpfYC9tNUwvUTE4xda5+UdSSpIrdCPiE5EHK+WXxoRvYj4h4iYi76LIuKBiHgkIm6ttqtVkySNz6qhHxHbgXuB5y8u/Sbw1sy8FvgxYA9wGzCfma8G3hgRlwxQkySNSZ0z/eeAm4GzAJn5nsz8ZrXuR4DvAl3gaFX7HDA9QE2SNCar3sjNzLMAEfED9Yi4Gfh6Zn4nIrYCp6pVp4EO/Z8M6tSWjjsLzAJ0Oh16vd5ADT1vcXGx8WPbxl7aZ1L6AHtpq7XqpdGrdyJiF/B7wHVVaRHYApwBpqrP69Z+QGbOAXMA09PT2fTutXfx22lSepmUPsBe2mqtehn41TvVNf6PAbdm5pmqPA/sr5b3AgsD1CRJY9LkTP8g8FLgUHXJ5730b/Qei4hrgJcDn6d/GadOTZI0JrVDPzO71b/vAt61dH1EXE//LP4PMvM54PGatTVx8tQZ3nLwwbUafkULd7xuXfYrSasZ2TtyM/M7nHtlzkA1SdJ4+I5cSSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0JakgtUI/IjoRcbxavigiHoiIRyLi1mFrkqTxWTX0I2I7cC+wtSrdBsxn5quBN0bEJUPWJEljEpm58gYRLwICuD8zuxHxKeBgZn4jIg4Cnwd+t2ktM/9xyf5mgVmATqdz1ZEjRxo19tTpMzz5bKOHDm3P5dtGOt7i4iJTU1MjHXO9TEovk9IH2EtbDdPLzMzMfGZOL7du82oPzsyzABHxfGkrcKpaPg10hqwt3d8cMAcwPT2d3W53tSku69B993PnyVXbWxMLB7ojHa/X69H069A2k9LLpPQB9tJWa9VLkxu5i8CWanmqGmOYmiRpTJqE7jywv1reCywMWZMkjUmT6x/3Asci4hrg5fSv1Z8aoiZJGpPaZ/qZ2a3+fRy4HngEuC4znxumNtJuJEkranSnMzO/AxwdVU2SNB7eSJWkghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klSQgUM/IrZHxLGIeCwi/rqqHY6IExFx+3nb1apJksanyZn+m4H7MnMauCQifh/YlJn7gF0RsTsibqpTG1kXkqRaIjMHe0DEAeAVwAeAB4CvAQ9k5rGIuAXYAlwBPLRaLTPvWWb8WWAWoNPpXHXkyJFGjT11+gxPPtvooUPbc/m2kY63uLjI1NTUSMdcL5PSy6T0AfbSVsP0MjMzM1+dmL/A5gbj/TPwOuC3gW8CFwOnqnWngSuBrTVrL5CZc8AcwPT0dHa73QZThEP33c+dJ5u0N7yFA92Rjtfr9Wj6dWibSellUvoAe2mrteqlyeWd9wK/lZnvA74F/Cr9M3mAqWrMxZo1SdIYNQne7cCeiNgEvAq4A9hfrdsLLADzNWuSpDFqcv3jT4F7gJcBJ4C/AI5HxA7gRuBqIGvWJEljNPCZfmZ+ITN/IjOnMvP6zDwLdIFHgZnMPFO3NqomJEn1jOROZ2Y+DRxtUpMkjY83UyWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBWkcehHxN0R8YZq+XBEnIiI289bX6smSRqfRqEfEdcAL8nMByLiJmBTZu4DdkXE7rq1kXUhSaolMnOwB0RcBJwEjgH/BPws8FBmHouIW4AtwBV1apl5zzLjzwKzAJ1O56ojR440auyp02d48tlGDx3ansu3jXS8xcVFpqamRjrmepmUXialD7CXthqml5mZmfnMnF5u3eYG4/0a8A3gz4DbgLcBh6t1p4Erga3AqRq1F8jMOWAOYHp6OrvdboMpwqH77ufOk03aG97Cge5Ix+v1ejT9OrTNpPQyKX2AvbTVWvXSJBWvAOYy84mI+BvgZ+ifyQNM0b9ktFizJkkaoybB+x/Armp5GtgJ7K8+3wssAPM1a5KkMWpypn8Y+HB1Xf4ioAt8KiJ2ADcCVwMJHK9RkySN0cBn+pn5vcz85cx8TWbuy8zH6Qf/o8BMZp7JzLN1aqNqQpJUz0judGbm08DRJjVJ0vh4M1WSCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBGod+RHQi4kvV8uGIOBERt5+3vlZNkjQ+w5zp/zmwJSJuAjZl5j5gV0TsrlsbfvqSpEE0Cv2IuBZ4BngC6AJHq1UPA/sHqEmSxigyc7AHRFwMfAb4ReCTwH8CH8rMr0TEDcCVwO46tcy8Y5nxZ4FZgE6nc9WRI0caNfbU6TM8+Wyjhw5tz+XbRjre4uIiU1NTIx1zvUxKL5PSB9hLWw3Ty8zMzHxmTi+3bnOD8Q4Cd2fm/0UEwCKwpVo3Rf+nh7q1F8jMOWAOYHp6OrvdboMpwqH77ufOk03aG97Cge5Ix+v1ejT9OrTNpPQyKX2AvbTVWvXS5PLOdcDbIqIH/CTwBs5dqtkLLADzNWuSpDEa+FQ4M1/z/HIV/D8PHI+IHcCNwNVA1qxJksZoqNfpZ2Y3M8/Sv0n7KDCTmWfq1obZtyRpcCO56J2ZT3PulTkD1SRJ4+M7ciWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgqyPn9aSpI2gJ0HH1y3fX/ktVvXZFzP9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUkIFDPyK2RcSnI+LhiPhERFwcEYcj4kRE3H7edrVqkqTxaXKmfwC4KzNvAJ4AbgE2ZeY+YFdE7I6Im+rURtWEJKmegX+1cmbefd6nlwFvAv6y+vxhYD9wBXC0Ru3fB5+yJKmpxr9PPyL2AduBBeBUVT4NXAlsrVlbbtxZYBag0+nQ6/Uaza+zBd655/uNHjuspnO+kMXFxZGPuV4mpZdJ6QPsZSXrlSGwds9Lo9CPiBcDh4BfAt4BbKlWTdG/ZLRYs/YCmTkHzAFMT09nt9ttMkUO3Xc/d55cn78Rs3CgO9Lxer0eTb8ObTMpvUxKH2AvK3nLOv8RlbV4XprcyL0Y+Djw7sx8HJinf6kGYC/9M/+6NUnSGDU5Ff51+pdm3hMR7wHuAd4cETuAG4GrgQSO16hJksZo4DP9zPyrzNyemd3q416gCzwKzGTmmcw8W6c2qiYkSfWM5KJ3Zj7NuVfmDFSTJI2P78iVpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUkM3rPQFNhp0HH6y13Tv3fJ+31Ny2joU7XjeysaQSjD30I+Iw8HLgwcx8/7j3L210db/B1lX3G7HfYCfDWC/vRMRNwKbM3Afsiojd49y/JJUuMnN8O4v4EPBQZh6LiFuALZl5z5JtZoHZ6tMfB/6t4e4uBb7beLLtYi/tMyl9gL201TC9vCwzL1tuxbgv72wFTlXLp4Erl26QmXPA3LA7iojHMnN62HHawF7aZ1L6AHtpq7XqZdyv3lkEtlTLU+uwf0kq2rhDdx7YXy3vBRbGvH9JKtq4L+98EjgeETuAG4Gr13BfQ18iahF7aZ9J6QPspa3WpJex3sgFiIjtwPXA5zLzibHuXJIKN/bQlyStH2+kSlJBNtSvYYiIz3LhOX87M980zvkMY1J6mZQ+YLJ6WU6dd8NvlHfMrzbPiNgM/Ff1AXBbZp4c4xRri4gO8LeZec0F1l8E/B3wYuBwZn54qB1m5ob5AK5bYd0vANuATwMPA58ALl5h+8PACeD2NvZS/dsBjq8yzuXAt4Fe9XHZBu3jIuAB4BHg1hY/J6seN/S/cfz3ec/JnvXoZ8mcbgI+Ui1/GNjdZJs2fNTs5UrgA+s91xq9bAceAr64wjbvAP6wWj4GXDLMPift8s4B4K7MvAF4AnjtchtthF8HUd3wvpf+G9pW8irgjzOzW33879rPrr4B+rgNmM/MVwNvjIhL1nxyAxrguHkl8LHznpM2nGF2gaPV8sOce+n0oNu0QZfV53k18PqI+EJEHK7O/NvoOeBm4OwK23Q51+/ngKHesDVRoZ+Zd2fm31efXgY8dYFNu7T/4K5zMED/4P6NiPhiRPzJ2k9rYHX76DLCA3uNdKl33LQxcJa+G77TcJs2qDPPf6X/k9tP0/8p8ufGNLeBZObZzDyzymYjfV4mKvSfFxH7gO2Z+egFNmn9wV3zYID+5awu8FPAvoh45ZpObEAD9NH654T6c2xj4NR5N/xGecd8nXl+NTP/p1p+DGjdT/MDGOnz0tYntbGIeDFwCLh1hc02ysFdx79k5vcy8zngS2zcg3sjPCd159jGwKnzbviN8o75OvP8aETsjYhN9O/3fWVMc1sLI31e2vgfq7GIuBj4OPDuzHx8hU03ysFdx2ci4kcj4oeBG4CvrfeEGtoIz0ndObYxcD4JvDki7gJ+Bfh6RCx91cvSbUb7i/tHp04v7wM+CnwZOJGZnx3zHBuJiGsj4u1LyvcCfxQRH6T/iqXPD7WT9b57PeCd7tVevfNW4GnOvWri5uqL9P4l276I/n/Eu4BvAtva1st5y73zlq8F3r5k2xngW8BXl67bYH28DPg68EH6l0c2ta2X5Y6bCxxfr6iej5P0b7KPtY8VethOPyRfMsw2bfjYKPMcYb87qn6HzqoN9Y7ciPg4/Ru0y/lyZv7OAGOt66+DGGUv62nEz8kO+mfSn8l69wFGqk4v633cSMPaUKEvSRrORF3TlyStzNCXpIIY+pJUEENfkgpi6EtSQf4fP1wXsycH7K8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "validation.location_traffic_convenience.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the stop words\n",
    "file = r'.\\zh_stopwords.txt'\n",
    "def get_stopwords(path):\n",
    "    stop_words = []\n",
    "    with open(path, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "        for line in lines:\n",
    "            stop_words.append(line.strip())\n",
    "    return stop_words\n",
    "stop_words = get_stopwords(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['$', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '?', '_', '“', '”']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trans2simp(line):\n",
    "    line = Converter('zh-hans').convert(line)\n",
    "    return line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'我们吃饭去了，测试句子。'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans2simp('我們吃饭去了，测试句子。')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = train.append(validation).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 120000 entries, 0 to 119999\n",
      "Data columns (total 22 columns):\n",
      "id                                          120000 non-null int64\n",
      "content                                     120000 non-null object\n",
      "location_traffic_convenience                120000 non-null int64\n",
      "location_distance_from_business_district    120000 non-null int64\n",
      "location_easy_to_find                       120000 non-null int64\n",
      "service_wait_time                           120000 non-null int64\n",
      "service_waiters_attitude                    120000 non-null int64\n",
      "service_parking_convenience                 120000 non-null int64\n",
      "service_serving_speed                       120000 non-null int64\n",
      "price_level                                 120000 non-null int64\n",
      "price_cost_effective                        120000 non-null int64\n",
      "price_discount                              120000 non-null int64\n",
      "environment_decoration                      120000 non-null int64\n",
      "environment_noise                           120000 non-null int64\n",
      "environment_space                           120000 non-null int64\n",
      "environment_cleaness                        120000 non-null int64\n",
      "dish_portion                                120000 non-null int64\n",
      "dish_taste                                  120000 non-null int64\n",
      "dish_look                                   120000 non-null int64\n",
      "dish_recommendation                         120000 non-null int64\n",
      "others_overall_experience                   120000 non-null int64\n",
      "others_willing_to_consume_again             120000 non-null int64\n",
      "dtypes: int64(21), object(1)\n",
      "memory usage: 20.1+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>location_traffic_convenience</th>\n",
       "      <th>location_distance_from_business_district</th>\n",
       "      <th>location_easy_to_find</th>\n",
       "      <th>service_wait_time</th>\n",
       "      <th>service_waiters_attitude</th>\n",
       "      <th>service_parking_convenience</th>\n",
       "      <th>service_serving_speed</th>\n",
       "      <th>price_level</th>\n",
       "      <th>price_cost_effective</th>\n",
       "      <th>price_discount</th>\n",
       "      <th>environment_decoration</th>\n",
       "      <th>environment_noise</th>\n",
       "      <th>environment_space</th>\n",
       "      <th>environment_cleaness</th>\n",
       "      <th>dish_portion</th>\n",
       "      <th>dish_taste</th>\n",
       "      <th>dish_look</th>\n",
       "      <th>dish_recommendation</th>\n",
       "      <th>others_overall_experience</th>\n",
       "      <th>others_willing_to_consume_again</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>\"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                            content  \\\n",
       "0   0  \"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，...   \n",
       "\n",
       "   location_traffic_convenience  location_distance_from_business_district  \\\n",
       "0                            -2                                        -2   \n",
       "\n",
       "   location_easy_to_find  service_wait_time  service_waiters_attitude  \\\n",
       "0                     -2                 -2                         1   \n",
       "\n",
       "   service_parking_convenience  service_serving_speed  price_level  \\\n",
       "0                           -2                     -2           -2   \n",
       "\n",
       "   price_cost_effective  price_discount  environment_decoration  \\\n",
       "0                    -2               1                      -2   \n",
       "\n",
       "   environment_noise  environment_space  environment_cleaness  dish_portion  \\\n",
       "0                 -2                 -2                    -2            -2   \n",
       "\n",
       "   dish_taste  dish_look  dish_recommendation  others_overall_experience  \\\n",
       "0          -2          1                   -2                          1   \n",
       "\n",
       "   others_willing_to_consume_again  \n",
       "0                               -2  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut(string):\n",
    "    '''\n",
    "    # remove stop_words seems to be not very well\n",
    "    string = trans2simp(string)\n",
    "    w_list = jieba.cut(string)\n",
    "    nw_list = []\n",
    "    \n",
    "    for w in w_list:\n",
    "        if w not in stop_words:\n",
    "            nw_list.append(w)\n",
    "    return ' '.join(w for w in nw_list)\n",
    "    '''\n",
    "    string = trans2simp(string).replace('\\n', '')\n",
    "    w_list = jieba.cut(string)\n",
    "    return ' '.join(w for w in w_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/24/2019 02:10:13 - DEBUG - jieba -   Building prefix dict from the default dictionary ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading model from cache C:\\Users\\Dracu\\AppData\\Local\\Temp\\jieba.cache\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/24/2019 02:10:13 - DEBUG - jieba -   Loading model from cache C:\\Users\\Dracu\\AppData\\Local\\Temp\\jieba.cache\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading model cost 0.621 seconds.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/24/2019 02:10:14 - DEBUG - jieba -   Loading model cost 0.621 seconds.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/24/2019 02:10:14 - DEBUG - jieba -   Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\" 第三次 参加 大众 点评 网 霸王餐 的 活动 。 这家 店 给 人 整体 感觉 一般 。 首先 环境 只能 算 中等 ， 其次 霸王餐 提供 的 菜品 也 不是 很多 ， 当然 商家 为了 避免 参加 霸王餐 吃不饱 的 现象 ， 给 每桌 都 提供 了 至少 六份 主食 ， 我们 那桌 都 提供 了 两份 年糕 ， 第一次 吃火锅 会 在 桌上 有 这么 多 的 主食 了 。 整体 来说 这家 火锅店 没有 什么 特别 有 特色 的 ， 不过 每份 菜品 分量 还是 比较 足 的 ， 这点 要 肯定 ！ 至于 价格 ， 因为 没有 看 菜单 不 了解 ， 不过 我 看 大众 有 这家 店 的 团购 代金券 ， 相当于 7 折 ， 应该 价位 不会 很 高 的 ！ 最后 还是 要 感谢 商家 提供 霸王餐 ， 祝 生意兴隆 ， 财源 广进 \"'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cut(data['content'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['content'] = data['content'].apply(cut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>location_traffic_convenience</th>\n",
       "      <th>location_distance_from_business_district</th>\n",
       "      <th>location_easy_to_find</th>\n",
       "      <th>service_wait_time</th>\n",
       "      <th>service_waiters_attitude</th>\n",
       "      <th>service_parking_convenience</th>\n",
       "      <th>service_serving_speed</th>\n",
       "      <th>price_level</th>\n",
       "      <th>price_cost_effective</th>\n",
       "      <th>price_discount</th>\n",
       "      <th>environment_decoration</th>\n",
       "      <th>environment_noise</th>\n",
       "      <th>environment_space</th>\n",
       "      <th>environment_cleaness</th>\n",
       "      <th>dish_portion</th>\n",
       "      <th>dish_taste</th>\n",
       "      <th>dish_look</th>\n",
       "      <th>dish_recommendation</th>\n",
       "      <th>others_overall_experience</th>\n",
       "      <th>others_willing_to_consume_again</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>\" 吼吼 吼 ， 萌死 人 的 棒棒糖 ， 中 了 大众 点评 的 霸王餐 ， 太 可爱 了...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>\" 第三次 参加 大众 点评 网 霸王餐 的 活动 。 这家 店 给 人 整体 感觉 一般 ...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                            content  \\\n",
       "0   0  \" 吼吼 吼 ， 萌死 人 的 棒棒糖 ， 中 了 大众 点评 的 霸王餐 ， 太 可爱 了...   \n",
       "1   1  \" 第三次 参加 大众 点评 网 霸王餐 的 活动 。 这家 店 给 人 整体 感觉 一般 ...   \n",
       "\n",
       "   location_traffic_convenience  location_distance_from_business_district  \\\n",
       "0                            -2                                        -2   \n",
       "1                            -2                                        -2   \n",
       "\n",
       "   location_easy_to_find  service_wait_time  service_waiters_attitude  \\\n",
       "0                     -2                 -2                         1   \n",
       "1                     -2                 -2                        -2   \n",
       "\n",
       "   service_parking_convenience  service_serving_speed  price_level  \\\n",
       "0                           -2                     -2           -2   \n",
       "1                           -2                     -2            0   \n",
       "\n",
       "   price_cost_effective  price_discount  environment_decoration  \\\n",
       "0                    -2               1                      -2   \n",
       "1                    -2               1                       0   \n",
       "\n",
       "   environment_noise  environment_space  environment_cleaness  dish_portion  \\\n",
       "0                 -2                 -2                    -2            -2   \n",
       "1                  0                  0                     0             1   \n",
       "\n",
       "   dish_taste  dish_look  dish_recommendation  others_overall_experience  \\\n",
       "0          -2          1                   -2                          1   \n",
       "1          -2         -2                   -2                          1   \n",
       "\n",
       "   others_willing_to_consume_again  \n",
       "0                               -2  \n",
       "1                               -2  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    \" 吼吼 吼 ， 萌死 人 的 棒棒糖 ， 中 了 大众 点评 的 霸王餐 ， 太 可爱 了...\n",
       "1    \" 第三次 参加 大众 点评 网 霸王餐 的 活动 。 这家 店 给 人 整体 感觉 一般 ...\n",
       "2    \" 4 人 同行   点 了 10 个 小吃 榴莲 酥   榴莲 味道 不足   松软   ...\n",
       "3    \" 之前 评价 了 莫名其妙 被删   果断 继续 差评 ！   换 了 菜单   价格 更...\n",
       "4    \" 出乎意料 地 惊艳 ， 椰子 鸡 清热 降火 ， 美容 养颜 ， 大大 满足 了 爱 吃...\n",
       "Name: content, dtype: object"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['content'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-6d0f13d39998>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mr'./data/train.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1500\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mr'./data/val.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'data' is not defined"
     ]
    }
   ],
   "source": [
    "data.to_csv(r'./data/train.csv', index=False, encoding='utf_8')\n",
    "data[len(data)-1500:len(data)].to_csv(r'./data/val.csv', index=False, encoding='utf_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Always use this\n",
    "data = pd.read_csv('./data/train.csv')\n",
    "label_cols = data.columns.values.tolist()[2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test small dataset\n",
    "s_train = data[:2000].to_csv(r'./data/s_train.csv', index=False, encoding='utf_8')\n",
    "s_valid = data[2000:2200].to_csv(r'./data/s_valid.csv', index=False, encoding='utf_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "df['label'] = data.columns.values.tolist()[2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(r'./labels/labels.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('corpus.txt', 'w', encoding='utf-8') as f:\n",
    "    for i in range(len(data)):\n",
    "        f.write(data['train_content'][i] + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = word2vec.Text8Corpus(\"corpus.txt\")\n",
    "model = word2vec.Word2Vec(sentences, size=300, window=10, min_count=1, workers=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"train.model\")\n",
    "#model_2 = word2vec.Word2Vec.load(\"text8.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import apex\n",
    "from fastai.text import *\n",
    "import datetime\n",
    "run_start_time = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = {\n",
    "    \"run_text\": \"my_test\",\n",
    "    \"max_seq_length\": 256, #512\n",
    "    \"do_lower_case\": True,\n",
    "    \"train_batch_size\": 4, #16\n",
    "    \"learning_rate\": 6e-5,\n",
    "    \"num_train_epochs\": 12.0,\n",
    "    \"warmup_proportion\": 0.002,\n",
    "    \"local_rank\": -1,\n",
    "    \"gradient_accumulation_steps\": 1,\n",
    "    \"fp16\": False,\n",
    "    \"loss_scale\": 32 #128\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOG_PATH = Path('logs/')  \n",
    "MODEL_PATH = Path('models/') \n",
    "\n",
    "if not LOG_PATH.exists():\n",
    "    LOG_PATH.mkdir()\n",
    "import logging\n",
    "\n",
    "logfile = str(LOG_PATH/'log-{}-{}.txt'.format(run_start_time, args[\"run_text\"]))\n",
    "\n",
    "logging.basicConfig(\n",
    "    level=logging.INFO,\n",
    "    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "    datefmt='%m/%d/%Y %H:%M:%S',\n",
    "    handlers=[\n",
    "        logging.FileHandler(logfile),\n",
    "        logging.StreamHandler(sys.stdout)\n",
    "    ])\n",
    "\n",
    "logger = logging.getLogger()\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "if torch.cuda.device_count() > 1:\n",
    "    multi_gpu = True\n",
    "else:\n",
    "    multi_gpu = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fast_bert.data_cls import BertDataBunch\n",
    "from fast_bert.learner_cls import BertLearner\n",
    "from fast_bert.metrics import accuracy_multilabel\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path(r'./data/')     \n",
    "LABEL_PATH = Path(r'./labels/')  \n",
    "\n",
    "BERT_PRETRAINED_PATH = Path(r'./chinese_L-12_H-768_A-12/')\n",
    "#BERT_PRETRAINED_PATH = 'bert-base-chinese'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenizer = BertTokenizer.from_pretrained(BERT_PRETRAINED_PATH, do_lower_case=args['do_lower_case'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/28/2019 22:23:40 - INFO - transformers.tokenization_utils -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt from cache at C:\\Users\\Dracu\\.cache\\torch\\transformers\\8a0c070123c1f794c42a29c6904beb7c1b8715741e235bee04aca2c7636fc83f.9b42061518a39ca00b8b52059fd2bede8daa613f8a8671500e518a8c29de8c00\n",
      "11/28/2019 22:23:40 - INFO - root -   Loading features from cached file data\\cache\\cached_bert_train_multi_label_256\n",
      "11/28/2019 22:23:40 - INFO - root -   Loading features from cached file data\\cache\\cached_bert_dev_multi_label_256\n"
     ]
    }
   ],
   "source": [
    "# create DataBunch object\n",
    "databunch = BertDataBunch(DATA_PATH, LABEL_PATH, \n",
    "                          tokenizer='bert-base-chinese', \n",
    "                          train_file='s_train.csv', \n",
    "                          val_file='s_valid.csv', \n",
    "                          label_file='labels.csv',\n",
    "                          text_col='content',\n",
    "                          label_col=label_cols,\n",
    "                          batch_size_per_gpu=args['train_batch_size'], \n",
    "                          max_seq_length=args['max_seq_length'], \n",
    "                          multi_gpu=multi_gpu, \n",
    "                          multi_label=True,\n",
    "                          model_type='bert'\n",
    "                         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = [{'name': 'accuracy_multilabel', 'function': accuracy_multilabel}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/28/2019 21:23:43 - INFO - root -   {'run_text': 'my_test', 'max_seq_length': 256, 'do_lower_case': True, 'train_batch_size': 4, 'learning_rate': 6e-05, 'num_train_epochs': 12.0, 'warmup_proportion': 0.002, 'local_rank': -1, 'gradient_accumulation_steps': 1, 'fp16': False, 'loss_scale': 32}\n"
     ]
    }
   ],
   "source": [
    "logger.info(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/28/2019 22:23:52 - INFO - transformers.configuration_utils -   loading configuration file chinese_L-12_H-768_A-12\\config.json\n",
      "11/28/2019 22:23:52 - INFO - transformers.configuration_utils -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 20,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "11/28/2019 22:23:52 - INFO - transformers.modeling_utils -   loading weights file chinese_L-12_H-768_A-12\\pytorch_model.bin\n",
      "11/28/2019 22:23:54 - INFO - transformers.modeling_utils -   Weights of BertForMultiLabelSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "11/28/2019 22:23:54 - INFO - transformers.modeling_utils -   Weights from pretrained model not used in BertForMultiLabelSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    }
   ],
   "source": [
    "# create a Learner object\n",
    "learner = BertLearner.from_pretrained_model(databunch,\n",
    "                                            pretrained_path=BERT_PRETRAINED_PATH,\n",
    "                                            metrics=metrics,\n",
    "                                            device=device,\n",
    "                                            logger=logger,\n",
    "                                            output_dir=MODEL_PATH,\n",
    "                                            finetuned_wgts_path=None,\n",
    "                                            warmup_steps=500,\n",
    "                                            multi_gpu=multi_gpu,\n",
    "                                            is_fp16=args['fp16'],\n",
    "                                            loss_scale=args['loss_scale'],\n",
    "                                            multi_label=True,\n",
    "                                            logging_steps=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models\\tensorboard\n",
      "11/28/2019 22:24:11 - INFO - root -   ***** Running training *****\n",
      "11/28/2019 22:24:11 - INFO - root -     Num examples = 400\n",
      "11/28/2019 22:24:11 - INFO - root -     Num Epochs = 6\n",
      "11/28/2019 22:24:11 - INFO - root -     Total train batch size (w. parallel, distributed & accumulation) = 4\n",
      "11/28/2019 22:24:11 - INFO - root -     Gradient Accumulation steps = 1\n",
      "11/28/2019 22:24:11 - INFO - root -     Total optimization steps = 600\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='6', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/6 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='100', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 12.00 MiB (GPU 0; 6.00 GiB total capacity; 4.60 GiB already allocated; 11.14 MiB free; 114.13 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-60-389c80931eee>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m             \u001b[0mvalidate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m             \u001b[0mschedule_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"warmup_cosine\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m             optimizer_type=\"lamb\")\n\u001b[0m",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\fast_bert\\learner_cls.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, validate, schedule_type, optimizer_type)\u001b[0m\n\u001b[0;32m    219\u001b[0m                     \u001b[0minputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'token_type_ids'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    220\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 221\u001b[1;33m                 \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    222\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m  \u001b[1;31m# model outputs are always tuple in pytorch-transformers (see doc)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\fast_bert\\modeling.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, token_type_ids, attention_mask, labels, position_ids, head_mask)\u001b[0m\n\u001b[0;32m    152\u001b[0m         \u001b[1;31m# pdb.set_trace()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    153\u001b[0m         outputs = self.bert(input_ids, position_ids=position_ids, token_type_ids=token_type_ids,\n\u001b[1;32m--> 154\u001b[1;33m                             attention_mask=attention_mask, head_mask=head_mask)\n\u001b[0m\u001b[0;32m    155\u001b[0m         \u001b[0mpooled_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    156\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask)\u001b[0m\n\u001b[0;32m    625\u001b[0m         encoder_outputs = self.encoder(embedding_output,\n\u001b[0;32m    626\u001b[0m                                        \u001b[0mextended_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 627\u001b[1;33m                                        head_mask=head_mask)\n\u001b[0m\u001b[0;32m    628\u001b[0m         \u001b[0msequence_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    629\u001b[0m         \u001b[0mpooled_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpooler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msequence_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask)\u001b[0m\n\u001b[0;32m    346\u001b[0m                 \u001b[0mall_hidden_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mall_hidden_states\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 348\u001b[1;33m             \u001b[0mlayer_outputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer_module\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    349\u001b[0m             \u001b[0mhidden_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    350\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask)\u001b[0m\n\u001b[0;32m    324\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    325\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 326\u001b[1;33m         \u001b[0mattention_outputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mattention\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    327\u001b[0m         \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mattention_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m         \u001b[0mintermediate_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintermediate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattention_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_tensor, attention_mask, head_mask)\u001b[0m\n\u001b[0;32m    281\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    282\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_tensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 283\u001b[1;33m         \u001b[0mself_outputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_tensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    284\u001b[0m         \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mattention_output\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mself_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m  \u001b[1;31m# add attentions if we output them\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask)\u001b[0m\n\u001b[0;32m    220\u001b[0m         \u001b[1;31m# This is actually dropping out entire tokens to attend to, which might\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    221\u001b[0m         \u001b[1;31m# seem a bit unusual, but is taken from the original Transformer paper.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 222\u001b[1;33m         \u001b[0mattention_probs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattention_probs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    224\u001b[0m         \u001b[1;31m# Mask heads if we want to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\modules\\dropout.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     52\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 54\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtraining\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     55\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\torch\\nn\\functional.py\u001b[0m in \u001b[0;36mdropout\u001b[1;34m(input, p, training, inplace)\u001b[0m\n\u001b[0;32m    805\u001b[0m     return (_VF.dropout_(input, p, training)\n\u001b[0;32m    806\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 807\u001b[1;33m             else _VF.dropout(input, p, training))\n\u001b[0m\u001b[0;32m    808\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 12.00 MiB (GPU 0; 6.00 GiB total capacity; 4.60 GiB already allocated; 11.14 MiB free; 114.13 MiB cached)"
     ]
    }
   ],
   "source": [
    "# train the model\n",
    "learner.fit(epochs=6,\n",
    "            lr=args['learning_rate'],\n",
    "            validate=True,\n",
    "            schedule_type=\"warmup_cosine\",\n",
    "            optimizer_type=\"lamb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/28/2019 21:43:10 - INFO - root -   Writing example 0 of 1\n",
      "11/28/2019 21:43:10 - INFO - root -   Saving features into cached file data\\cache\\cached_bert_test_multi_label_256\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[[('others_overall_experience', 0.6227706074714661),\n",
       "  ('dish_taste', 0.46760639548301697),\n",
       "  ('service_waiters_attitude', 0.3008595407009125),\n",
       "  ('dish_portion', 0.2891949415206909),\n",
       "  ('environment_noise', 0.26900535821914673),\n",
       "  ('location_easy_to_find', 0.2471245676279068),\n",
       "  ('others_willing_to_consume_again', 0.23873268067836761),\n",
       "  ('environment_cleaness', 0.23855119943618774),\n",
       "  ('price_discount', 0.22974546253681183),\n",
       "  ('location_distance_from_business_district', 0.22523802518844604),\n",
       "  ('environment_decoration', 0.22421284019947052),\n",
       "  ('location_traffic_convenience', 0.19162552058696747),\n",
       "  ('service_serving_speed', 0.19049598276615143),\n",
       "  ('environment_space', 0.18446438014507294),\n",
       "  ('price_level', 0.1692456156015396),\n",
       "  ('service_wait_time', 0.158110111951828),\n",
       "  ('dish_look', 0.15700635313987732),\n",
       "  ('price_cost_effective', 0.1169365867972374),\n",
       "  ('service_parking_convenience', 0.10419387370347977),\n",
       "  ('dish_recommendation', 0.0947360023856163)]]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.predict_batch(['吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，大众点评给了我这个土老冒一个见识的机会。看介绍棒棒糖是用德国糖做的，不会很甜，中间的照片是糯米的，能食用，真是太高端大气上档次了，还可以买蝴蝶结扎口，送人可以买礼盒。我是先打的卖家电话，加了微信，给卖家传的照片。等了几天，卖家就告诉我可以取货了，去大官屯那取的。虽然连卖家的面都没见到，但是还是谢谢卖家送我这么可爱的东西，太喜欢了，这哪舍得吃啊。'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}